package main;

/**
 * Created by dell on 2016/5/5.
 */

import edu.cn.nlsde.tmfst.textCrawler.CommentCrawler;
import edu.cn.nlsde.tmfst.textCrawler.TmallCommentCrawler;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import java.sql.SQLException;
import java.util.HashMap;


public class Main {

    public static void main(String[] args) throws SQLException {

//        CommentCrawler cc = new TmallCommentCrawler();
//        cc.get_Comment_list("https://detail.tmall.com/item.htm?spm=a222t.8063993.1387691862.12.oAjwdV&id=541826951378&acm=lb-zebra-164656-989409.1003.4.859249&scene=taobao_shop&scm=1003.4.lb-zebra-164656-989409.ITEM_541826951378_859249&sku_properties=10004:1162302684;5919063:6536025;12304035:3222911");
//        edu.cn.nlsde.tmfst.crawler.handler.handler_JD.crawler_JD(10000);
//        String url = "https://detail.tmall.com/item.htm?spm=a222t.7785392.113.19.2zbznM&id=526449276263&acm=lb-zebra-20612-413860.1003.4.511128&scm=1003.4.lb-zebra-20612-413860.ITEM_526449276263_511128&sku_properties=5919063:6536025;12304035:116177";
////        handler_TM.crawler_comment(url);
//        List<Comment> list = Comment.get_comment("526449276263");
//        List<String> corpus = new ArrayList<>();
//        for (Comment item : list) {
////            System.out.println(item.comment_content);
//            corpus.add(item.comment_content);
//        }
//
//        List<List<String>> tmp = data_pre_process_Chinese.get_segment_Corpus(corpus);
//            for(List<String> oneTopic:tmp){
//            for (String item:oneTopic){
//                System.out.print(item);
//                System.out.print(" ");
//            }
//            System.out.println();
//        }

//        List<List<String>> topics = LDA.get_LDA_Topics(corpus,20,2000,0.1,0.01);
//        for(List<String> oneTopic:topics){
//            for (String item:oneTopic){
//                System.out.print(item);
//                System.out.print(" ");
//            }
//            System.out.println("\n");
//        }

        Request req = new Request("http://item.m.jd.com/newComments/newCommentsDetail.json");
        req.setMethod("POST");
        HashMap<String, Object> pairs = new HashMap<>();
        pairs.put("wareId", "2131674");
        pairs.put("offset", 3);
        pairs.put("num", 10);
        pairs.put("checkParam", "LUIPPTP");
        pairs.put("evokeType", "");
        pairs.put("type", 0);
        req.setExtras(pairs);
        Request req2 = new Request("https://m.jd.com");
        Request[] ho = new Request[2];
        ho[0] = req2;
        ho[1] = req;
        Spider.create(new PageProcessor() {
            @Override
            public void process(Page page) {
                System.out.println(page);
            }

            @Override
            public Site getSite() {
                return Site.me()
                        .setSleepTime(500)
                        .addHeader("Accept-Encoding", "gzip, deflate")
                        .addHeader("Accept-Language", "zh,zh-TW;q=0.8,en;q=0.6,en-US;q=0.4,zh-CN;q=0.2")
                        .addHeader("Cookie", "JAMCookie=true")
                        .addHeader("X-Requested-With", "XMLHttpRequest")
                        .addHeader("Accept", "application/json")
                        .addHeader("Host", "item.m.jd.com")
                        .addHeader("Origin", "http://item.m.jd.com")
                        .addHeader("Content-Type", "application/x-www-form-urlencoded")
                        .addHeader("Referer", "http://item.m.jd.com/product/2131674.html")
                        .addHeader("Cookie", "sid=9f17f36253fc69c69e02609918bd4bcb;")
                        .setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");
            }
        })
                .addRequest(ho)
                .thread(1)
                .run();


    }
}
